package cn.q3c.download;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.log4j.Logger;

import cn.keyvalue.ccf.common.object.Hyperlink;
import cn.keyvalue.ccf.common.object.ListPage;
import cn.keyvalue.ccf.common.util.ObjectUtils;
import cn.q3c.download.base.BaseClear;
import cn.q3c.download.base.BaseSpider;
import cn.q3c.download.commons.ConfigProject;
import cn.q3c.download.commons.Configuration;
import cn.q3c.download.commons.DownloadLog;
import cn.q3c.download.commons.DownloadUtils;
import cn.q3c.download.thread.DownloadFileThread;

import org.apache.commons.configuration.PropertiesConfiguration;

/**
 * 系统后台启动
 * 
 * @author wangxinning
 * 
 */
public class Run {

	private static Logger logger = Logger.getLogger(Run.class);

	// 爬虫对象map
	private static Map<String, BaseSpider> spiderMap = new HashMap<String, BaseSpider>();
	// 爬去时间记录
	private static Map<String, Long> spiderTimes = new HashMap<String, Long>();

	// 线程池
	private static ThreadPoolExecutor threadPoolExecutor = null;

	public static void main(String[] args) {

		// 0.线程池初始化
		threadPoolExecutor = new ThreadPoolExecutor(3, 5,
				DateUtils.MILLIS_PER_MINUTE, TimeUnit.MILLISECONDS,
				new ArrayBlockingQueue<Runnable>(3));
		threadPoolExecutor
				.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());

		// 1.无限循环中
		while (true) {
			// 2.获取全部项目(动态更新)
			List<ConfigProject> projectList = Configuration.getInstance()
					.getConfigProjectList();

			// 3.循环项目
			for (ConfigProject project : projectList) {
				processProject(project);
			}

			// 4.一轮完成，停顿一会
			try {
				Thread.sleep(DateUtils.MILLIS_PER_SECOND * Configuration.getInstance().getWaitTime());
				logger.info("...");
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	private static BaseSpider getSpider(ConfigProject project) {
		String key = project.getName() + project.getSpiderclass();

		BaseSpider spider = spiderMap.get(key);
		if (spider == null) {
			try {
				spider = (BaseSpider) (Class.forName(project.getSpiderclass())
						.newInstance());
				//spider.setUrls(project.getUrls());// 传参数
				spider.setConfigProject(project);
				spiderMap.put(key, spider);
				
				//加载配置文件中的信息，内容均为系统生成
				String configFileStr = DownloadUtils.getProjectConfigPath(project
						.getName()) + "config.properties";
				File configFile = new File( configFileStr );
				if( configFile.exists() )
				{
					PropertiesConfiguration config = new PropertiesConfiguration(configFile);
					String lastUrl = config.getString("current.url","");
					spider.setLastUrl(lastUrl);
				}
				
			} catch (Exception e) {
				logger.error("动态加载解析爬虫异常 " + project.getSpiderclass(), e);
			}
		}

		return spider;
	}

	/**
	 * 处理下载项目的总方法，可能迁出 建立线程池，循环加入链接，无需理会线程池是否结束
	 * 
	 * @param project
	 */
	private static void processProject(ConfigProject project) {
		
		// 0.非活动状态直接跳出
		if( !project.isActive() )
		{
			logger.debug("非激活状态，不进行下载处理  " + project.getName() + " "
					+ project.getDescription());
			return;
		}
		
		// 1.获取爬虫部分
		BaseSpider spider = getSpider(project);

		// 2.爬取
		String charset = spider.getCharset();
		int waitMinute = project.getTimelimit();// 每轮间隔，时间未到不运行

		// 3.判断是否到时间
		long lastTime = spiderTimes.get(project.getName()) == null ? 0L
				: spiderTimes.get(project.getName());
		if (System.currentTimeMillis() - lastTime < waitMinute
				* DateUtils.MILLIS_PER_MINUTE) {
			// 没到下一次抓取时间
			logger.debug("继续等待下一次抓取 " + project.getName() + " "
					+ project.getSpiderclass());

		} else {
			// 抓取
			List<ListPage> listPageList = spider.getSpiderHyperlinkList();
			
			if (!ObjectUtils.isEmptyList(listPageList)) {

				long start = System.currentTimeMillis();
				
				int hyperlinkCount = 0;
				for (ListPage listPage : listPageList) {

					//1.保存列表页面
					Hyperlink parentHyperlink = new Hyperlink();
					parentHyperlink.setUrl(listPage.getUrl());
					
					File parentFile = null;
					
					//下载父页面，只下载，不记录日志
					try
					{
						parentFile = DownloadUtils.getSaveNextFile(project.getName(),spider, false);
						if( StringUtils.isNotBlank(listPage.getUrl()) )
						{							
							DownloadFileThread downloadFileThread = new DownloadFileThread(
									parentHyperlink, parentFile, charset, project.getType());
							threadPoolExecutor.execute(downloadFileThread);
						}
					}
					catch( Exception e )
					{
						logger.error("父页面计算下一个保存文件名出现异常", e);
					}
					
					//2.保存子页面
					hyperlinkCount += listPage.getHyperlinkList().size();
					//for( Hyperlink hyperlink:listPage.getHyperlinkList() )//倒序
					for( int i = listPage.getHyperlinkList().size()-1; i >= 0; i--  )
					{
						Hyperlink hyperlink = listPage.getHyperlinkList().get(i);
						try {
							File nextFile = DownloadUtils.getSaveNextFile(project.getName(),spider,true);
							DownloadFileThread downloadFileThread = new DownloadFileThread(
									hyperlink, nextFile, charset, project.getType());
							threadPoolExecutor.execute(downloadFileThread);
							spider.setLastUrl(hyperlink.getUrl());
							addProjectLog(project, hyperlink, nextFile, parentHyperlink, parentFile);
						} catch (Exception e) {
							logger.error("计算下一个保存文件名出现异常", e);
						}
					}
				}
				
				long end = System.currentTimeMillis();

				logger.info( project.getDescription() + "(" + project.getName() + ") 下载任务完成，下载文件数：" + hyperlinkCount + "," + (end-start)/DateUtils.MILLIS_PER_SECOND + "s" );
			
			} else {

				logger.debug("没有需要抓取的链接  " + project.getName() + " "
						+ project.getSpiderclass());

			}
			
			// 记录开始处理时间
			spiderTimes.put(project.getName(), System.currentTimeMillis());
		}

		// 4.删除文件
		try {
			BaseClear clear = new BaseClear();
			clear.setProjectName(project.getName());
			clear.setSaveLimitHour(project.getSaveLimitHour());
			clear.clear();
			logger.debug( project.getDescription() + "(" + project.getName() + ") 删除历史数据任务完成" );
		} catch (Exception e) {
			logger.error("清除过期文件异常" + project.getName(), e);
		}
	}


	/**
	 * 记录项目日志，每小时生成一个，在config/logs/年/月/日 记录 可能存在异常，没有本地文件的情况
	 * 
	 * 最外层存一个最新200个的日志
	 * 
	 * @throws IOException
	 */
	public static void addProjectLog(ConfigProject project,
			Hyperlink hyperlink, File file, Hyperlink parentHyperlink, File parentFile) throws IOException {
		String logPath = DownloadUtils.getProjectConfigPath(project.getName())
				+ "logs" + File.separator;
		String newLogFileStr = logPath + "new.log";
		int limit = 200;// new.log中保存最新日志记录数，有时会有用，因为足够小

		long current = System.currentTimeMillis();
		String logFile = DownloadUtils.getProjectLogFile(project.getName(),
				new Date(current));

		DownloadLog dLog = new DownloadLog();
		dLog.setDate(new Date(current));
		dLog.setFile(file);
		dLog.setName(StringUtils.trimToEmpty(hyperlink.getName()));
		dLog.setUrl(StringUtils.trimToEmpty(hyperlink.getUrl()));
		dLog.setParentUrl(parentHyperlink!=null?parentHyperlink.getUrl():"");
		dLog.setParentFile(parentFile);
		String logText = dLog.toString();
		FileUtils.writeStringToFile(new File(logFile),
				logText + System.getProperty("line.separator"), true);
		
		// 记录new.log
		File newLogFile = new File(newLogFileStr);
		if (!newLogFile.exists()) {
			FileUtils.touch(newLogFile);
		}

		List<String> contentList = FileUtils.readLines(newLogFile);

		if (!ObjectUtils.isEmptyList(contentList)) {
			if (contentList.size() > limit) {
				contentList.remove(0);
			}
		} else {
			contentList = new ArrayList<String>();
		}

		contentList.add(logText);
		FileUtils.writeLines(newLogFile, contentList);
	}
}
